<?php
// composer下载方式
// 先使用composer命令下载：
// composer require owner888/phpspider
// 引入加载器
//require './vendor/autoload.php';

// GitHub下载方式
require_once __DIR__ . '/../autoloader.php';
    use phpspider\core\db;
    use phpspider\core\phpspider;
    use phpspider\core\requests;
    
    /* Do NOT delete this comment */
/* 不要删除这段注释 */
/**
 * 实例化阿里云oos
 * @return object 实例化得到的对象
 * @throws \OSS\Core\OssException
 */
function new_oss(){
    require_once __DIR__."/../../simplewind/Core/Library/Vendor/Alioss/autoload.php";
    $config=array(
        'KEY_ID'             => 'LTAItUsYcUr43aWQ', // 阿里云oss key_id
        'KEY_SECRET'         => 'dseeyXS9xmDqIzehY2YI91JosbdGQe', // 阿里云oss key_secret
        'END_POINT'          => 'http://oss-cn-beijing.aliyuncs.com', // 阿里云oss endpoint 华北2
        'BUCKET'             => 'dbcc0'  // bucken 名称
    );
    $oss=new \OSS\OssClient($config['KEY_ID'],$config['KEY_SECRET'],$config['END_POINT']);
    return $oss;
}

/**
 * 上传文件到oss并删除本地文件
 * @param  string $path 文件路径
 * @return bool 是否上传
 * @throws \OSS\Core\OssException
 */
function oss_upload($path){
    // 获取配置项
    $bucket='dbcc0';
    // 先统一去除左侧的.或者/ 再添加./
    $oss_path=ltrim($path,'./');
    $path='./'.$path;
    if (file_exists($path)) {
        // 实例化oss类
        $oss=new_oss();
        // 上传到oss
        $oss->uploadFile($bucket,$oss_path,$path);
        // 如需上传到oss后 自动删除本地的文件 则删除下面的注释
        unlink($path);
        return true;
    }
    return false;
}
$configs = array(
    'name' => 'tuchong',
    'log_show' => true,
    'tasknum' => 4,
    'save_running_state' => true,
    'domains' => array(
        'stock.tuchong.com',
    ),
    'scan_urls' => array(
        'https://stock.tuchong.com/'
    ),
    'list_url_regexes' => array(
        "https://stock.tuchong.com/api/free/search/?term=\s+&page=\d+",
        //"https://pixabay.com/zh/editors_choice/?media_type=video&pagi=\d+"
    ),
    'content_url_regexes' => array(
        "https://stock.tuchong.com/free/image?imageId=\d+",
        //"https://pixabay.com/zh/videos/none-\d+",
    ),
    'max_try' => 5,
    //'proxies' => array(
        //'http://H784U84R444YABQD:57A8B0B743F9B4D2@proxy.abuyun.com:9010'
    //),
    //'export' => array(
        //'type' => 'csv',
        //'file' => '../data/qiushibaike.csv',
    //),
    //'export' => array(
        //'type'  => 'sql',
        //'file'  => '../data/qiushibaike.sql',
        //'table' => 'content',
    //),
    //链接数据库表
    'export' => array(
        'type' => 'db',
        'table' => 'cco_images',
    ),
    //链接数据库
    'db_config' => array(
        'host'  => '127.0.0.1',
        'port'  => 3306,
        'user'  => 'root',
        'pass'  => '15726818351',
        'name'  => 'ios',
    ),
    //redis
    'queue_config' => array(
        'host'      => '127.0.0.1',
        'port'      => 6379,
        'pass'      => '',
        'db'        => 5,
        'prefix'    => 'tuchong',
        'timeout'   => 30,
    ),
    //过滤要执行的数据
    'fields' => array(
        array(
            'name' => "title",
            'selector' => "//*[@id=\"index\"]/div[2]/div[2]/a",
            'required' => true,
            'repeated' =>true,//是否获取多数据
        ),
        array(
            'name' => "pic",
            'selector' => "//*[@id='image-detail']/div/div[2]/img",
            'required' => true,
        ),
        array(
            'name' => "label",
            'selector' => "//*[@id=\"index\"]/div[2]/div[2]/a",
            'required' => true,
            'repeated' =>true,//是否获取多数据
        ),
        array(
            'name' => "protocol",
            'selector' => "//*[@id='image-detail']/div/div[1]/div[3]/div[1]/span[2]",
            'required' => true,
        ),
        array(
            'name' => "create_time",
            'selector' => "//*[@id='image-detail']/div/div[1]/div[2]/div[3]/span[2]",
            'required' => true,
        ),
        array(
            'name' => "width",
            'selector' => "//*[@id='image-detail']/div/div[2]/img",
            'required' => true,
        ),
        array(
            'name' => "height",
            'selector' => "//*[@id='image-detail']/div/div[2]/img",
            'required' => true,
        ),
        array(
            'name' => "type",
            'selector' => "//*[@id='image-detail']/div/div[2]/img",
            'required' => true,
        ),
        array(
            'name' => "username",
            'selector' => "//*[@id='image-detail']/div/div[2]/img",
            'required' => true,
        )
    ),
);


$spider = new phpspider($configs);
$spider->on_start = function($phpspider)
{
    $db_config = $phpspider->get_config("db_config");
    //print_r($db_config);
    //exit;
    // 数据库连接
    db::set_connect('default', $db_config);
    db::init_mysql();
    //$login_url="https://pixabay.com/zh/accounts/login/?next=https://pixabay.com";
    //$params = array(
    //    "username" => "w_1121562734",
    //    "password" => "wjb64800891",
    //);
    //// 发送登录请求
    //requests::post($login_url, $params);
    //// 登录成功后本框架会把Cookie保存到www.waduanzi.com域名下，我们可以看看是否是已经收集到Cookie了
    //$cookies = requests::get_cookies("tuchong.com");
     //requests对象自动收集Cookie，访问这个域名下的URL会自动带上
    //print_r($cookies);  // 可以看到已经输出Cookie数组结构
    requests::set_useragent(array(
        "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/",
        "Opera/9.80 (Android 3.2.1; Linux; Opera Tablet/ADR-1109081720; U; ja) Presto/2.8.149 Version/11.10",
        "Mozilla/5.0 (Android; Linux armv7l; rv:9.0) Gecko/20111216 Firefox/9.0 Fennec/9.0",
    ));
};
    
$spider->on_scan_page = function($page, $content, $phpspider)
{
    $term = array(
        "tv",
        "电视",
        "盒子",
        "投影",
        "语音",
        "遥控",
        "高清",
        "客厅",
        "音箱",
        "音响",
        "小米",
        "乐视",
        "海信",
        "智能",
        "AI",
        "品牌",
        "客服",
        "足球",
        "游戏",
        "app",
        "ps4",
        "xobx",
        "企业家",
        "雷军",
        "创维",
        "酷开",
        "设计",
        "创意",
        "广告"
    );
    foreach($term as $key  => $value){
        if(empty($key)){//获取当前最大的页面
            $list=requests::get("https://stock.tuchong.com/api/free/search/?term=".$value);
            $con=json_decode($list,true);
            $i_num=ceil($con['data']['totalCount']/100);
        }
        for ($i = 1; $i < $i_num; $i++)
        {
            //通过搜索信息获取详情页id
            $cont=requests::get("https://stock.tuchong.com/api/free/search/?term=".$value."&page=".$i);
            $cont_arr=json_decode($cont,true);
            foreach($cont_arr['data']['hits'] as $cont_key => $cont_v){
                //添加内容页信息
                $phpspider->add_url("https://stock.tuchong.com/free/image?imageId=".$cont_v['imageId'], array(),0,true);
            }
        }
    }
};
$spider->on_extract_field = function($fieldname, $data, $page) 
{
    if ($fieldname == 'pic')
    {
        $str=strpos($data,'http');
        if($str === false){
            $url="http:".$data;
        }else{
            $url=$data;
        }
       
        //返回文件的基本信息
        $pathinfo = pathinfo($url);
        $fileext =  $pathinfo['extension'];
        if (strtolower($fileext) == 'jpeg')
        {
            $fileext = 'jpg';
        }
        if(strlen($fileext)>3){
            $fileext=substr($fileext, 0,3);
        }
        // 以纳秒为单位生成随机数
        $filename = uniqid().".".$fileext;
        $date=date("Ymd");
        $dirs=dirname(__FILE__)."/../../uploads/{$date}";
        // 在data目录下生成图片
        $filepath =  $dirs."/".$filename;
        if(!is_dir($dirs)){
            mkdir($dirs, 0777, true);
        }
        // 用系统自带的下载器wget下载
        exec("wget -q {$url} -O {$filepath}");
        $img_path = "uploads/{$date}/{$filename}";
        oss_upload($img_path);
        // 替换成真是图片url
        $data = $img_path;
    }elseif($fieldname == 'title'){
        if(is_array($data)){
            $data=implode(',', $data);
        }
    }
    elseif($fieldname == 'label'){
        if(is_array($data)){
            $data=implode(',', $data);
        }
    }
    elseif ($fieldname == 'create_time')
    {
        // 用当前采集时间戳作为发布时间
        if($data){
            date_default_timezone_set('Asia/Shanghai');
            $data=str_replace(".", "-", $data);
            $data=strtotime($data)+rand(0,86399);
        }else{
            $data = time();
        }
    
    }elseif ($fieldname == 'protocol'){// 把当前内容页URL替换上面的field
        $str=strpos($data,'CC0');
        if($str !== false){
            $data='CC0';
        }else{
            return false;
        }
    }
    elseif($fieldname == 'username'){
        $data='tuchong';
    }
    return $data;
};
    
$spider->on_extract_page = function($page,$data){
    $sql = "Select Count(*) As `count` From `cco_images` Where `title`='{$data['title']}'";
    $row = db::get_one($sql);
    //if(!$row['count']){
    //    $str=strpos($data['protocol'],'CC0');
    //    if($str){
    //        $data['protocol']='CC0';
    //        db::insert("cco_images", $data);
    //
    //    }
    //}
    if(!$row['count']){
        
        $cc0=strpos($data['protocol'],'CC0');
        if($cc0 !== false){
            $data['protocol']='CC0';
    
            $str=strpos($data['width'],'http');
            if($str === false){
                $url="http:".$data['width'];
            }else{
                $url=$data['width'];
            }
            $pathinfo = pathinfo($url);
            $fileext =  $pathinfo['extension'];
            if (strtolower($fileext) == 'jpeg')
            {
                $fileext = 'jpg';
            }
            if(strlen($fileext)>3){
                $fileext=substr($fileext, 0,3);
            }
            // 以纳秒为单位生成随机数
            $filename = uniqid().".".$fileext;
            $date=date("Ymd");
            $dirs=dirname(__FILE__)."/../../uploads/{$date}";
            echo $dirs;
            // 在data目录下生成图片
            $filepath =  $dirs."/".$filename;
            if(!is_dir($dirs)){
                mkdir($dirs, 0777, true);
            }
            echo "\r".$filepath."\r";
            // 用系统自带的下载器wget下载
            exec("wget -q {$url} -O {$filepath}");
            $img_path = "uploads/{$date}/{$filename}";
            $pathinfo = getimagesize("./".$img_path);
            unlink("./".$img_path);
            $data['width'] = $pathinfo[0];
            $data['height'] = $pathinfo[1];
            $data['type'] = $fileext;
            
            return $data;
        }else{
            return false;
        }
    }else{
        return false;
    }
};
$spider->start();


